﻿using System;
using System.Collections.Generic;
using System.Linq;

namespace Comdiv.QWeb.Serialization.BxlParser {
	public class BxlTokenizer {
		public int BufferSize = 2048;
		private char[] buffer;
		private int bufferidx;
		private char c;
		private int col;
		private BxlTokenType currentType;
		private bool donotextractelements;
		private int endidx;
		private int exprnest;
		private string filename;
		private int idx;
		private bool incomment;
		private bool inexpression;
		private bool inliteral;
		private string input;
		private bool instring;
		private BxlToken last;
		private int line;
		private List<BxlToken> result;
		private int skips;
		private int startcol;
		private int startidx;
		private int startline;
		private string stringstart;
		private bool tripplestring;
		private bool wascolon;
		private bool wasdelimiter;
		private bool wasnsstart;

		public bool AllowNonClosedStringsAndExpressions { get; set; }
		public bool HasOpenedValueAtStart { get; set; }
		public bool HasOpenedAttributeAtStart { get; set; }
		public bool HasOpenedStringAtStart { get; set; }
		public bool HasOpenedExpressionAtStart { get; set; }
		public int ExpressionNestLevelAtStart { get; set; }
		public bool OpenedStartIsClosed { get; set; }

		public BxlTokenizer NoElements {
			get {
				donotextractelements = true;
				return this;
			}
		}

		public bool HasOpenedValueAtEnd { get; set; }

		public bool HasOpenedAttributeAtEnd { get; set; }

		public int ExpressionNestLevelAtEnd { get; set; }

		public bool HasOpenedExpressionAtEnd { get; set; }

		public bool HasOpenedStringAtEnd { get; set; }

		public void SetInitialState(int statecode) {
			AllowNonClosedStringsAndExpressions = true;
			HasOpenedAttributeAtStart = false;
			HasOpenedValueAtStart = false;
			HasOpenedExpressionAtStart = false;
			HasOpenedStringAtStart = false;


			var attrvalselector = statecode/10000;
			if (attrvalselector > 0) {
				if (attrvalselector == 1) {
					HasOpenedValueAtStart = true;
				}
				else {
					HasOpenedAttributeAtStart = true;
				}
				var strexprselector = (statecode%10000)/1000;
				if (strexprselector == 0)
					throw new BxlParserException("invalid string or expression selector", new BxlLexInfo(filename, line, col));
				if (strexprselector == 1) {
					HasOpenedStringAtStart = true;
				}
				else {
					HasOpenedExpressionAtStart = true;
					var nest = statecode%1000;
					if (0 == nest) {
						throw new BxlParserException("not closed expressions must have nest level more than zero",
						                             new BxlLexInfo(filename, line, col));
					}
					ExpressionNestLevelAtStart = nest;
				}
			}
		}

		public int GetFinishState() {
			if (!AllowNonClosedStringsAndExpressions) return 0;
			var result = 0;
			if (HasOpenedValueAtEnd) {
				result += 10000;
			}
			if (HasOpenedAttributeAtEnd) {
				result += 20000;
			}
			if (HasOpenedStringAtEnd) {
				result += 1000;
			}
			if (HasOpenedExpressionAtEnd) {
				result += 2000;
				result += ExpressionNestLevelAtEnd;
			}
			if (0 == result) result = 1; //indicate keep AllowNonClosed
			return result;
		}

		public BxlToken[] Tokenize(string input, string filename = "") {
			this.filename = filename ?? "main";
			this.input = input;

			resetTokenizer();
			foreach (var _c in this.input) {
				if (InitIteration(_c)) continue;
				currentType = GetSymbolType(_c);
				if (ProcessComment()) continue;
				if (ProcessNewLine()) continue;
				if (ProcessWhiteSpace()) continue;
				if (ProcessEscape()) continue;
				if (ProcessExpression()) continue;
				if (ProcessString()) continue;
				if (ProcessDelimiter()) continue;
				if (ProcessLiteral()) continue;
			}
			FinalizeTokenizing();
			ExtractAttributes();
			if (!donotextractelements) {
				ExtractValues();
				ExtractElements();
			}
			var r = result.ToArray();
			if (AllowNonClosedStringsAndExpressions) {
				if (2 < r.Length) {
					var opencandidate = r.Reverse().Skip(1).First();
					if (opencandidate.IsValidOpenState()) {
						if (opencandidate.SubType.IsString() ||
						    (null != opencandidate.Right && opencandidate.Right.Type.IsString())
							) {
							HasOpenedStringAtEnd = true;
						}
						else {
							HasOpenedExpressionAtEnd = true;
							ExpressionNestLevelAtEnd = exprnest;
						}
						if (opencandidate.IsAttributeOpener()) {
							HasOpenedAttributeAtEnd = true;
						}
						else {
							HasOpenedValueAtEnd = true;
						}
					}
					else if (opencandidate.IsInValidOpenState()) {
						//treat such cases as anonym attributes
						opencandidate.SubType = opencandidate.Type;
						opencandidate.Type = BxlTokenType.AttributeStart;
						HasOpenedAttributeAtEnd = true;
						if (opencandidate.SubType.IsString()) {
							HasOpenedStringAtEnd = true;
						}
						else {
							HasOpenedExpressionAtEnd = true;
							ExpressionNestLevelAtEnd = exprnest;
						}
					}
				}
			}
			return r;
		}

		private void ExtractValues() {
			if (HasOpenedValueAtStart) {
				if (result.Count > 3)
					throw new BxlParserException("illegal element value construction", new BxlLexInfo(filename, line, col));
				var candidate = result[1];
				if (OpenedStartIsClosed) {
					candidate.SubType = candidate.Type;
					candidate.Type = BxlTokenType.ValueEnd;
				}
				else {
					candidate.SubType = candidate.Type;
					candidate.Type = BxlTokenType.ValuePart;
					HasOpenedValueAtEnd = true;
					if (candidate.SubType.IsString()) {
						HasOpenedStringAtEnd = true;
					}
					else {
						HasOpenedExpressionAtEnd = true;
						ExpressionNestLevelAtEnd = exprnest;
					}
				}
			}
			else {
				for (var i = result.Count - 1; i >= 0; i--) {
					if (result[i].ColonPrefixed) {
						var c = result[i];

						if (c.Type == BxlTokenType.NewLine) {
							continue;
						}


						if (c.IsValue() && c.Next.IsLineFinish()) {
							c.SubType = c.Type;
							c.Type = BxlTokenType.Value;
							continue;
						}
						if (AllowNonClosedStringsAndExpressions) {
							if (c.IsStartValue() && c.Next.IsLineFinish()) {
								c.SubType = c.Type;
								c.Impl = c.Standalone();
								c.Type = BxlTokenType.ValueStart;
								continue;
							}
						}
						throw new BxlParserException("illegal value declaration " + result[i].LexInfo, new BxlLexInfo(filename, line, col));
					}
				}
			}
		}


		private void ExtractElements() {
			if (HasOpenedAttributeAtStart || HasOpenedValueAtStart) return;
			for (var i = result.Count - 1; i >= 0; i--) {
				if (result[i].IsLineStart()) {
					var ec = result[i + 1];

					if (ec.Type == BxlTokenType.Literal) {
						ec.SubType = ec.Type;
						ec.Type = BxlTokenType.Element;
						ec.Name = ec.Value;
						if (ec.Prev.Type == BxlTokenType.Level) {
							ec.Level = ec.Prev.Level;
						}
					}
					else if (ec.Type == BxlTokenType.Attribute || ec.Type == BxlTokenType.Level || ec.Type == BxlTokenType.Value ||
					         ec.Type == BxlTokenType.Finish || ec.Type == BxlTokenType.NewLine) {
					}
					else if (ec.Type == BxlTokenType.String) {
						ec.SubType = ec.Type;
						ec.Type = BxlTokenType.Value;
					}
					else {
						throw new BxlParserException(
							"all lines must be start with element and it must be literal or lines can be attributes " +
							result[i].LexInfo, new BxlLexInfo(filename, line, col));
					}
				}
			}
		}

		private void ExtractAttributes() {
			for (var i = result.Count - 1; i > 0; i--) {
				if (result[i].IsAssigner) {
					var left = result[i];
					var right = result[i + 1];
					if (left.CanAssign(right)) {
						var attr = result[i];
						attr.Left = new BxlToken {Type = result[i].Type, Value = result[i].Value};
						attr.Left.LexInfo = attr.LexInfo.Clone();
						attr.Right = right;
						attr.LexInfo.CharIndex = left.LexInfo.CharIndex;
						attr.LexInfo.Length = (right.LexInfo.CharIndex + right.LexInfo.Length) - left.LexInfo.CharIndex;
						attr.Type = BxlTokenType.Attribute;
						attr.Name = left.Value;
						attr.AttrValue = right.Value;
						right.Next.Prev = attr;
						result.RemoveAt(i + 1);
						if (attr.Prev.Type == BxlTokenType.Level) {
							attr.Level = attr.Prev.Level;
						}
						if (AllowNonClosedStringsAndExpressions) {
							if (right.IsStartValue()) {
								attr.Type = BxlTokenType.AttributeStart;
							}
						}
					}
					else {
						throw new BxlParserException(
							"cannot generate attribute from " + left.Type + " and " + right.Type + result[i].LexInfo,
							new BxlLexInfo(filename, line, col));
					}
				}
				if (1 == i && !result[i].ColonPrefixed) {
					if (HasOpenedAttributeAtStart && HasOpenedStringAtStart && result[i].Type == BxlTokenType.StringPart) {
						result[i].SubType = BxlTokenType.StringPart;
						result[i].Type = BxlTokenType.AttributePart;
					}
					else if (HasOpenedAttributeAtStart && HasOpenedExpressionAtStart &&
					         result[i].Type == BxlTokenType.ExpressionPart) {
						result[i].SubType = BxlTokenType.ExpressionPart;
						result[i].Type = BxlTokenType.AttributePart;
					}
					else if (HasOpenedAttributeAtStart && HasOpenedStringAtStart &&
					         result[i].Type == BxlTokenType.StringEnd) {
						result[i].SubType = BxlTokenType.StringEnd;
						result[i].Type = BxlTokenType.AttributeEnd;
					}
					else if (HasOpenedAttributeAtStart && HasOpenedExpressionAtStart &&
					         result[i].Type == BxlTokenType.ExpressionEnd) {
						result[i].SubType = BxlTokenType.ExpressionEnd;
						result[i].Type = BxlTokenType.AttributeEnd;
					}
				}
			}
		}

		private void FinalizeTokenizing() {
			if (inliteral) {
				appendliteral();
			}
			if (AllowNonClosedStringsAndExpressions) {
				if (inexpression) {
					if (HasOpenedExpressionAtStart && !OpenedStartIsClosed) {
						appendexpression(notclosed: true, ispart: true);
					}
					else {
						appendexpression(notclosed: true);
					}
				}
				if (instring && tripplestring) {
					if (HasOpenedStringAtStart && !OpenedStartIsClosed) {
						appendstring(notclosed: true, ispart: true);
					}
					else {
						appendstring(notclosed: true);
					}
				}
			}
			else {
				if (inexpression || exprnest > 0) {
					throw new BxlParserException("not closed expression, started at " + filename + ":" + startline + ":" + startcol,
					                             new BxlLexInfo(filename, line, col));
				}
			}
			if (instring) {
				throw new BxlParserException("not closed string, started at " + filename + ":" + startline + ":" + startcol,
				                             new BxlLexInfo(filename, line, col));
			}
			while (result.Last().IsWs()) {
				result.RemoveAt(result.Count - 1);
			}

			result.Add(new BxlToken {Type = BxlTokenType.Finish});
			result[0].Next = result[1];
			result[result.Count - 1].Prev = result[result.Count - 2];
		}


		private bool ProcessLiteral() {
//////////////////////////////////////////////////////////////////////////////////////
			/// LITERAL SUPPORT
			/// //////////////////////////////////////////////////////////////////////////////////

			//try append literal
			if (inliteral) {
				//NOTE: now any symbol not processed above can be in literal
				//if (currentType.CanBeInLiteral()) {
				endidx = idx - 1;
				buffer[bufferidx++] = c;
				return true;
				//}
				//throw new BxlParserException("illegal symbol in literal " + filename + ":" + line + ":" + col, new BxlLexInfo(filename, line, col));
			}

			//if no any mode and letter symbol - try start literal
			if (!inliteral && currentType.CanBeStartOfLiteral()) {
				wasdelimiter = false;
				inliteral = true;
				bufferidx = 0;
				buffer[bufferidx++] = c;
				startcol = col;
				startidx = idx - 1;
				endidx = idx - 1;
				startline = line;
				return true;
			}
			return false;
		}

		private bool ProcessDelimiter() {
/////////////////////////////////////////////////////////////////////
			/// SPECIAL SYMBOLS SUPPORT
			/// /////////////////////////////////////////////////////////////////

			if (currentType.IsSpecial()) {
				if (currentType == BxlTokenType.Colon) {
					if (inliteral && !wasnsstart && read(1) == ":") {
						endidx = idx - 1;
						buffer[bufferidx++] = ':';
						wasnsstart = true;
						return true;
					}
					if (wasnsstart) {
						if (read(1) == ":") {
							throw new BxlParserException("illegal tripple namespace delimiter", new BxlLexInfo(filename, line, col));
						}
						endidx = idx - 1;
						buffer[bufferidx++] = ':';
						wasnsstart = false;
						return true;
					}
				}
				if (inliteral) {
					appendliteral();
				}
				if (currentType == BxlTokenType.Colon) {
					wascolon = true;
					wasdelimiter = true;
					return true;
				}
				if (currentType == BxlTokenType.Comma) {
					wasdelimiter = true;
					return true;
				}
				if (currentType == BxlTokenType.Assign) {
					wasdelimiter = true;
					if (last.Type != BxlTokenType.Literal && last.Type != BxlTokenType.String) {
						throw new BxlParserException("cannot assign to " + last.Type, new BxlLexInfo(filename, line, col));
					}
					last.IsAssigner = true;
					return true;
				}


				if (wascolon) {
					throw new BxlParserException("was colon", new BxlLexInfo(filename, line, col));
				}
				append(currentType, c.ToString());
				return true;
			}

			return false;
		}

		private bool ProcessString() {
/////////////////////////////////////////////////////////////////////
			/// STRING SUPPORT (we must process string delimiters)
			/// suppose that escapes (including quots will be already processed
			/// /////////////////////////////////////////////////////////////////


			//process symbols in string which are collected in buffer
			if (instring && !(BxlTokenType.Quot == currentType || BxlTokenType.Apos == currentType)) {
				//при нахождении в строке и с простыми символами все просто
				buffer[bufferidx++] = c;
				return true;
			}
			//if quot is another than current string quot all is simple too
			if (instring && (BxlTokenType.Quot == currentType || BxlTokenType.Apos == currentType) && c.ToString() != stringstart) {
				//при нахождении в строке и с простыми символами все просто
				buffer[bufferidx++] = c;
				return true;
			}
			//if string is not tripple and it's closer we collapse string
			if (instring && (BxlTokenType.Quot == currentType || BxlTokenType.Apos == currentType) && c.ToString() == stringstart &&
			    !tripplestring) {
				appendstring();
				return true;
			}
			//close tripple string
			if (instring && (BxlTokenType.Quot == currentType) && tripplestring && read(2) == "\"\"") {
				skips++;
				skips++;
				appendstring();
				return true;
			}
			// not processed are usual chars
			if (instring && (BxlTokenType.Quot == currentType || BxlTokenType.Apos == currentType)) {
				buffer[bufferidx++] = c;
				return true;
			}

			//not in strings quots treats as open string constructs
			if (!instring && (BxlTokenType.Quot == currentType || BxlTokenType.Apos == currentType)) {
				if (inliteral)
					throw new BxlParserException("illegal character in literal " + filename + ":" + line + ":" + col,
					                             new BxlLexInfo(filename, line, col));
				if (!wasdelimiter && !wascolon && last.Type != BxlTokenType.Start && last.Type != BxlTokenType.Level &&
				    last.Type != BxlTokenType.NewLine)
					throw new BxlParserException("string must be precedet with delimeter " + filename + ":" + line + ":" + col,
					                             new BxlLexInfo(filename, line, col));
				instring = true;
				startcol = col;
				startline = line;
				startidx = idx - 1;
				stringstart = c.ToString();
				bufferidx = 0;
				//try to check tripple)
				if (BxlTokenType.Quot == currentType && read(2) == "\"\"") {
					tripplestring = true;
					skips++;
					skips++;
				}
				wasdelimiter = false;
				return true;
			}
			return false;
		}

		private bool ProcessExpression() {
/////////////////////////////////////////////////////////////////////
			/// EXPRESSION SUPPORT
			/// /////////////////////////////////////////////////////////////////

			if (inliteral && (BxlTokenType.OpenExpr == currentType || BxlTokenType.CloseExpr == currentType)) {
				buffer[bufferidx++] = c;
				return true;
			}

			if (inexpression && BxlTokenType.OpenExpr == currentType) {
				buffer[bufferidx++] = c;
				exprnest++;
				return true;
			}

			if (inexpression && BxlTokenType.CloseExpr == currentType) {
				buffer[bufferidx++] = c;
				exprnest--;
				if (exprnest == 0) {
					appendexpression();
				}
				return true;
			}

			if (inexpression) {
				buffer[bufferidx++] = c;
				return true;
			}

			if ((!instring && !inexpression && !inliteral) && BxlTokenType.OpenExpr == currentType) {
				inexpression = true;
				wasdelimiter = false;
				bufferidx = 0;
				buffer[bufferidx++] = c;
				startidx = idx;
				startcol = col;
				startline = line;
				exprnest ++;
				return true;
			}
			return false;
		}

		private bool ProcessEscape() {
//////////////////////////////////////////////////////////////////////
			/// ESCAPING SUPPORT - in string - reescapes symbols, in end of file
			/// or in not string it's BxlParserException
			/// /////////////////////////////////////////////////////////////////
			if (BxlTokenType.Escape == currentType) {
				if (idx == input.Length) throw new BxlParserException("escape at end of file", new BxlLexInfo(filename, line, col));
				if (!(instring || inexpression))
					throw new BxlParserException("escape not in string " + filename + ":" + line + ":" + col,
					                             new BxlLexInfo(filename, line, col));
				skips++;
				var next = input[idx];
				if (next == 'r') {
					buffer[bufferidx++] = '\r';
				}
				else if (next == 'n') {
					buffer[bufferidx++] = '\n';
				}
				else if (next == 't') {
					buffer[bufferidx++] = '\t';
				}
				else {
					buffer[bufferidx++] = next;
				}
				return true;
			}
			return false;
		}

		private bool ProcessWhiteSpace() {
//////////////////////////////////////////////////////////////////////
			/// White space handling
			/// in stirng we keep, in start of line we collect a level,
			/// in other cases we leave one SP ws,
			/// в начале триммируем
			/// //////////////////////////////////////////////////////////////////
			if (BxlTokenType.WhiteSpace == currentType) {
				if (instring || inexpression) {
//in strings all is very simple
					buffer[bufferidx++] = c;
					return true;
				}
				else if (inliteral) {
					appendliteral();
				}
				else if (last.Type == BxlTokenType.NewLine || last.Type == BxlTokenType.Start) {
// если только что был перенос строки то пробелы - это уровень строки
					var tc = 0;
					var wsc = 0;
					skips--;
					for (var i = idx - 1; i < input.Length; i++) {
						skips++;
						var nc = input[i]; // берем символ
						if (nc == ' ') {
							wsc++;
						}
						else if (nc == '\t') {
							tc++;
						}
						else {
							skips--;
							break;
						}
					}
					var lev = wsc/4 + tc;
					var token = new BxlToken {Type = BxlTokenType.Level};
					token.Level = lev;
					token.LexInfo = new BxlLexInfo {CharIndex = idx, Column = col, File = filename, Length = wsc + tc, Line = line};
					registerToken(token);

					last.Number = lev;
					last.LexInfo.Length = tc + wsc;
					return true;
				}

				wasdelimiter = true;
				return true;
				//append(BxlTokenType.WhiteSpace, sp);
			}
			return false;
		}

		private bool InitIteration(char _c) {
			c = _c;
			col++;
			idx++;
			//next checks if it must be skipped
			if (skips > 0) {
				skips--;
				return true;
			}
			return false;
		}

		private bool ProcessNewLine() {
			if (BxlTokenType.NewLine == currentType) {
				var newline = line; // заготовка на изменение строки лексической информации
				var val = c.ToString(); //значение переноса строки (промежуточный буфер)
				newline++; //в любом случае переводим строку
				for (var nlidx = idx; nlidx < input.Length; nlidx++) {
					//смотрим символы вперед на предмет переносов
					var nc = input[nlidx]; // берем символ
					var nt = GetSymbolType(nc); // получаем тип
					if (BxlTokenType.NewLine == nt) {
// и если продолжается перенос
						skips++; //помечаем его к пропуску
						val += nc; //заносим в буфер
						if (c == '\r' && nc != '\n') {
							newline++; //если это не \n в двух-символьных переносах, добавляем строчку
						}
						else if (c == '\n' && nc == '\n') {
							newline++;
						}
						continue;
					}
					break;
				}
				if (instring || inexpression) {
					//если мы были в строке, то просто наращиваем буфер строки для трипплеров 
					if (tripplestring || inexpression) {
						foreach (var __c in val) {
							buffer[bufferidx++] = __c;
						}
					}
					else {
						//или ошибку для обычных
						throw new BxlParserException("illegal newline in regular string at " + filename + ":" + line + ":" + col,
						                             new BxlLexInfo(filename, line, col));
					}
				}
				else {
					if (inliteral) {
						appendliteral();
					}


					//иначе формируем токен переноса строки

					if (last.Type != BxlTokenType.Start && last.Type != BxlTokenType.NewLine) {
						append(BxlTokenType.NewLine, val);
					}
				}
				line = newline; //выставляем новое значение для лексера
				col = 0;
				if (!(instring && tripplestring) && !inexpression) {
					//	wascolon = false;
					wasdelimiter = false;
				}
				return true;
			}
			return false;
		}

		private bool ProcessComment() {
			if (incomment) {
				if (BxlTokenType.NewLine == currentType) {
					incomment = false;
					//it it's new line we must  finish comment and process it as usual newline further
				}
				else {
					return true;
				}
				//any symbols except newline is skipped
			}
			else // on another hand we check if it's comment start
				if (BxlTokenType.Comment == currentType && !instring && !inexpression) {
					// comments was checked, the only variant that it's not comment is that it's int string
					incomment = true;
					return true;
				}
			return false;
		}

		private string read(int i) {
			var r = "";
			for (var c = idx; c < idx + i && c < input.Length; c++) {
				r += input[c];
			}
			return r;
		}

		private void appendexpression(bool notclosed = false, bool ispart = false) {
			var token = new BxlToken {Type = BxlTokenType.Expression, Value = new string(buffer, 0, bufferidx)};

			token.LexInfo = new BxlLexInfo
			                	{
			                		CharIndex = startidx,
			                		File = filename,
			                		Column = startcol,
			                		Length = idx - startcol + 1,
			                		Line = startline
			                	};
			if (notclosed) {
				token.Type = BxlTokenType.ExpressionStart;
				if (ispart) {
					token.Type = BxlTokenType.ExpressionPart;
				}
				token.CurrentNestLevel = exprnest;
				//token.LexInfo.Length--;
			}
			if (HasOpenedExpressionAtStart && !OpenedStartIsClosed && !notclosed) {
				token.Type = BxlTokenType.ExpressionEnd;
				token.LexInfo.CharIndex = 0;
				token.LexInfo.Length = idx;
				OpenedStartIsClosed = true;
			}
			bufferidx = 0;
			registerToken(token);
		}

		private void appendliteral() {
			var token = new BxlToken {Type = BxlTokenType.Literal, Value = new string(buffer, 0, bufferidx)};
			token.LexInfo = new BxlLexInfo
			                	{
			                		CharIndex = startidx,
			                		File = filename,
			                		Column = startcol,
			                		Length = endidx - startidx + 1,
			                		Line = startline
			                	};
			bufferidx = 0;
			registerToken(token);
		}

		private void appendstring(bool notclosed = false, bool ispart = false) {
			var token = new BxlToken {Type = BxlTokenType.String, Value = new string(buffer, 0, bufferidx)};
			token.LexInfo = new BxlLexInfo
			                	{
			                		CharIndex = startidx,
			                		File = filename,
			                		Column = startcol,
			                		Length = idx - startcol + 1,
			                		Line = startline
			                	};
			if (tripplestring) token.LexInfo.Length += 2;
			if (notclosed) {
				token.Type = BxlTokenType.StringStart;
				if (ispart) {
					token.Type = BxlTokenType.StringPart;
				}
				token.LexInfo.Length -= 2;
			}
			if (HasOpenedStringAtStart && !OpenedStartIsClosed && !notclosed) {
				token.Type = BxlTokenType.StringEnd;
				token.LexInfo.CharIndex = 0;
				token.LexInfo.Length = idx + 2;
				OpenedStartIsClosed = true;
			}
			bufferidx = 0;
			registerToken(token);
		}

		private void append(BxlTokenType t, string v) {
			var token = new BxlToken {Type = t, Value = v};
			token.LexInfo = new BxlLexInfo {CharIndex = idx, Column = col, File = filename, Length = 1, Line = line};
			registerToken(token);
		}

		private void registerToken(BxlToken token) {
			/*if (last.Type == BxlTokenType.WhiteSpace || last.Type == BxlTokenType.Comma) {
				result.RemoveAt(result.Count - 1);
			}*/
			if (wascolon) {
				token.ColonPrefixed = true;
				wascolon = false;
			}
			if (null != last) last.Next = token;
			result.Add(token);
			last = token;
			inexpression = false;
			inliteral = false;
			incomment = false;
			instring = false;
			tripplestring = false;
			startidx = 0;
			startcol = 0;
			startline = 0;
			//	buffer = new StringBuilder();
		}

		private void resetTokenizer() {
			instring = false;
			stringstart = "";
			startline = 0;
			startcol = 0;
			startidx = 0;
			tripplestring = false;
			inliteral = false;
			last = null;
			incomment = false;
			line = 1;
			col = 0;
			var size = Math.Min(BufferSize, input.Length);
			buffer = new char[size];
			bufferidx = 0;
			result = new List<BxlToken>(input.Length/10) {new BxlToken(BxlTokenType.Start)};

			last = result[0];
			idx = 0;
			skips = 0;
			inexpression = false;
			exprnest = 0;
			HasOpenedAttributeAtEnd = false;
			HasOpenedValueAtEnd = false;
			HasOpenedStringAtEnd = false;
			HasOpenedExpressionAtEnd = false;
			ExpressionNestLevelAtEnd = 0;
			if (HasOpenedAttributeAtStart || HasOpenedValueAtStart) {
				AllowNonClosedStringsAndExpressions = true;
			}
			if (HasOpenedExpressionAtStart) {
				inexpression = true;
				exprnest = ExpressionNestLevelAtStart;
				startcol = 1;
			}
			else if (HasOpenedStringAtStart) {
				instring = true;
				tripplestring = true;
				stringstart = "\"";
				startcol = 1;
			}
		}

		private BxlTokenType GetSymbolType(char _c) {
			switch (_c) {
				case '\r':
					goto case '\n';
				case '\n':
					return BxlTokenType.NewLine;
				case '\t':
					goto case ' ';
				case ' ':
					return BxlTokenType.WhiteSpace;
				case '#':
					return BxlTokenType.Comment;
				case '"':
					return BxlTokenType.Quot;
				case '\'':
					return BxlTokenType.Apos;
				case '\\':
					return BxlTokenType.Escape;
				case '=':
					return BxlTokenType.Assign;
				case '.':
					return BxlTokenType.NonLetter;
				case ':':
					return BxlTokenType.Colon;
				case ',':
					return BxlTokenType.Comma;
				case '1':
					goto case '0';
				case '2':
					goto case '0';
				case '3':
					goto case '0';
				case '4':
					goto case '0';
				case '5':
					goto case '0';
				case '6':
					goto case '0';
				case '7':
					goto case '0';
				case '8':
					goto case '0';
				case '9':
					goto case '0';
				case '0':
					return BxlTokenType.NonLetter;
				case '!':
					goto case '+';
				case '~':
					goto case '+';
				case '%':
					goto case '+';
				case '?':
					goto case '+';
				case '-':
					//return  BxlTokenType.Minus;
					goto case '+';
				case '*':
					goto case '+';
				case '/':
					goto case '+';
				case '$':
					goto case '+';
				case '@':
					goto default;
				case ';':
					goto case '+';
				case '<':
					goto case '+';
				case '>':
					goto case '+';
				case '&':
					goto case '+';
				case '^':
					goto case '+';
				case '|':
					goto case '+';
				case '(':
					return BxlTokenType.OpenExpr;
				case ')':
					return BxlTokenType.CloseExpr;
				case '{':
					goto case '+';
				case '}':
					goto case '+';
				case '[':
					goto case '+';
				case ']':
					goto case '+';
				case '`':
					goto case '+';
				case '+':
					return BxlTokenType.NonLetter;
				default:
					return BxlTokenType.Letter;
			}
		}
	}
}